#
# Copyright 2009 Red Hat, Inc. and/or its affiliates.
#
# Licensed to you under the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.  See the files README and
# LICENSE_GPL_v2 which accompany this distribution.
#

#FIXME: Some of the methods here contain SUDO in their names and some don't.
#This doesn't mean if the method will use sudo by default or not.
#Further more 'SUDO' never needs to be in a name of a method and should always be 'False' by default.
#Using sudo isn't something a method should do by default. As a rule of thumb. If you didn't
#have to you wouldn't use SUDO. This makes it the less desirable option thus making it the optional one.

#FIXME: A lot of methods here use DD. A smart thing would be to wrap DD in a method that does all
#the arg concatenation and stream handling once. Also most method when they fail don't tell why even
#though DD is more then happy to let you know. Exceptions thrown should contain DD's stderr output.

"""
Various storage misc procedures
"""
import shutil
import logging
import subprocess
import traceback
import types
import time
import os
import signal
import operator
import struct
import sys
from array import array
import string
import random
import threading
import gc
from glob import glob

from remoteFileHandler import remoteFileHandlerPool
ros = remoteFileHandlerPool

sys.path.append("../")
import constants
import storage_exception as se

IOUSER = "vdsm"
DIRECTFLAG='direct'
DATASYNCFLAG="fdatasync"
STR_UUID_SIZE = 36
UUID_HYPHENS = [8, 13, 18, 23]
OVIRT_NODE = False
MEGA = 1 << 20

SUDO_NON_INTERACTIVE_FLAG = "-n"
log = logging.getLogger('irs')

def propagateError(e):
    log.error(traceback.format_exc())
    raise e

def logException(e = None):
    """ Log exception trace to log file """
    if e:
        log.error(str(e))
    log.error(traceback.format_exc())

def panic(msg):
    log.error("Panic: %s" % (str(msg)))
    log.error(traceback.format_exc())
    os.killpg(0, 9)
    sys.exit(-3)

def getMounts():
    """
    returns a list of tuples represnting the mounts in this host.
    For each mount you have a tuple `` (resource, mount, type, options, freq, passno) ``.
    """
    rawMounts = open("/proc/mounts").readlines()
    parsedMounts = []
    for mount in rawMounts:
        parsedMounts.append(tuple(mount.split()))

    return parsedMounts

def execCmd(command, sudo=True, cwd=None, infile=None, outfile=None,
            shell=False, data=None, raw=False, logErr=True, printable=None):
    """
    Executes an external command, optionally via sudo.
    """
    if sudo:
        if isinstance(command, types.StringTypes):
            command = " " .join([constants.EXT_SUDO, SUDO_NON_INTERACTIVE_FLAG, command])
        else:
            command = [constants.EXT_SUDO, SUDO_NON_INTERACTIVE_FLAG] + command

    if not printable:
        printable = command
    log.debug("%s (cwd %s)", repr(subprocess.list2cmdline(printable)), cwd)

    # FIXME: if infile == None and data:
    if infile == None:
        infile = subprocess.PIPE

    if outfile == None:
        outfile = subprocess.PIPE

    disabledGcBlock.enter()
    try:
        p = subprocess.Popen(command, shell=shell, close_fds=True, cwd=cwd,
                        stdin=infile, stdout=outfile, stderr=subprocess.PIPE)
    finally:
        disabledGcBlock.exit()
    (out, err) = p.communicate(data)

    if out == None:
        # Prevent splitlines() from barfing later on
        out = ""

    # log.debug("<out> = %s; <err> = %s; <rc> = %d", out, repr(err), p.returncode)
    if p.returncode == 0:
        log.debug("SUCCESS: <err> = %s; <rc> = %d", repr(err), p.returncode)
    elif logErr:
        log.warning("FAILED: <err> = %s; <rc> = %d", repr(err), p.returncode)

    if not raw:
        out = out.splitlines()
        err = err.splitlines()

    return (p.returncode, out, err)


#FIXME : Returnes the opposit of what you would expect.
def pidPoll(pid):
    try:
        os.kill(pid, 0)
        return False
    except OSError:
        return True


def watchCmd(command, stop, idle, sudo=True, cwd=None, infile=None, outfile=None,
            shell=False, data=None):
    """
    Executes an external command, optionally via sudo with stop abilities.
    """
    if sudo:
        if isinstance(command, types.StringTypes):
            command = " " .join([constants.EXT_SUDO, SUDO_NON_INTERACTIVE_FLAG, command])
        else:
            command = [constants.EXT_SUDO, SUDO_NON_INTERACTIVE_FLAG] + command

    log.debug("WATCHCMD %s (cwd %s)", command, cwd)

    if infile == None:
        infile = subprocess.PIPE

    if outfile == None:
        outfile = subprocess.PIPE

    p = subprocess.Popen(command, shell=shell, close_fds=True, cwd=cwd,
                    stdin=infile, stdout=outfile, stderr=subprocess.PIPE)
    # poll returns None if process still exists, and returncode eitherwise
    while p.poll() is None:
        if stop():
            os.kill(p.pid, signal.SIGINT)
            log.info('Thread %u was killed.' % p.pid)
        log.info("idle %s" % idle)
        time.sleep(idle)

    (out, err) = p.communicate(data)

    if out == None:
        # Prevent splitlines() from barfing later on
        out = ""

    if p.returncode == 0:
        log.debug("SUCCESS: <err> = %s; <rc> = %d", repr(err), p.returncode)
    else:
        log.warning("FAILED: <err> = %s; <rc> = %d", repr(err), p.returncode)

    return (p.returncode, out.splitlines(), err.splitlines())

def randomStr(strLen):
    return "".join(random.sample(string.letters, strLen))

def fileexists(filename, writeable=False):
    # This function is workarround for a NFS issue where
    # sometimes os.exists/os.access fails due to NFS stale handle.
    # In such cases we should try again and stat the file
    check = os.R_OK
    if writeable:
        check |= os.W_OK
    if ros.access(filename, check):
        return True
    try:
        s = ros.stat(filename)
        if check & s[0] == check:
            return True
    except OSError:
        pass
    return False

def readfile(name):
    '''
    Read the content of the file using /bin/dd command
    '''
    cmd = [constants.EXT_DD, "iflag=%s" % DIRECTFLAG, "if=%s" % name]
    (rc, out, err) = execCmd(cmd, sudo=False)
    if rc:
        raise se.MiscFileReadException(name)
    return out

def readfileSUDO(name):
    """
    Read the content of the file using 'cat' command via sudo.
    """
    #Sometimes we think we need sudo but we don't
    if os.access(name, os.R_OK):
        try:
            lines = open(name, "r").readlines()
            return [l[:-1] for l in lines]
        except IOError, ex:
            if ex.errno != 13:
                raise

    cmd = [constants.EXT_CAT, name]
    (rc, out, err) = execCmd(cmd, sudo=True)
    if rc:
        raise se.MiscFileReadException(name)
    return out

def writefileSUDO(name, lines):
    """
    Write the 'lines' to the file using /bin/echo command via sudo
    """
    cmd = [constants.EXT_DD, "of=%s" % name]
    data = "".join(lines)
    (rc, out, err) = execCmd(cmd, data=data, sudo=True)
    if rc:
        log.warning("(misc.writefileSUDO) write failed with rc: %s, stderr: %s", rc, repr(err))
        raise se.MiscFileWriteException(name)

    if not validateDDBytes(err, len(data)):
        log.warning("(misc.writefileSUDO) write failed stderr: %s, data: %s", repr(err), len(data))
        raise se.MiscFileWriteException(name)

    return (rc,out)

def cleanupdir(dir, ignoreErrors = True):
    """
    Recursively remove all the files and directories in the given directory
    """
    shutil.rmtree(dir, ignore_errors=True)
    if not ignoreErrors and ros.lexists(dir):
        raise se.MiscDirCleanupFailure(dir)

def readblockSUDO(name, offset, size, sudo=False):
    '''
    Read (direct IO) the content of device 'name' at offset, size bytes
    '''

    # direct io must be aligned on block size boundaries
    if (size % 512) or (offset % 512):
        raise se.MiscBlockReadException(name, offset, size)

    left = size
    ret = ""
    baseoffset = offset

    while left > 0:
        (iounit, count, iooffset) = _alignData(left, offset)

        cmd = [constants.EXT_DD, "iflag=%s" % DIRECTFLAG, "skip=%d" % iooffset,
                "bs=%d" % iounit, "if=%s" % name, 'count=%s' % count]

        (rc, out, err) = execCmd(cmd, raw=True, sudo=sudo)
        if rc:
            raise se.MiscBlockReadException(name, offset, size)
        if not validateDDBytes(err.splitlines(), iounit*count):
            raise se.MiscBlockReadIncomplete(name, offset, size)

        ret += out
        left = left % iounit
        offset = baseoffset + size - left
    return ret.splitlines()


def validateDDBytes(ddstderr, size):
    log.debug("(validateDDBytes) err: %s, size: %s" % (ddstderr, size))
    try:
        size = int(size)
    except:
        return False

    if len(ddstderr) != 3:
        return False
    try:
        xferred = int(ddstderr[2].split()[0])
    except:
        return False

    if xferred != size:
        return False
    return True


def writeblockSUDO(name, offset, size, lines, sudo=False):
    '''
    Write (direct IO) the content of device 'name' at offset, size bytes
    '''
    # direct io must be aligned on block size boundaries
    if offset % 512:
        raise se.MiscBlockWriteException("%s: offset not aligned -" % (name) , offset, size)

    # align size to 512 to allow direct IO
    size = ((size+511) / 512) * 512 #FIXME: trusting '/' to round your result is considered bad practice. Using `(512 - size) % 512 + size` is safer.
    data = "".join(lines)
    if len(data) < size:
        data += "\0" * (size - len(data))
    elif len(data) > size:
        log.warning("(writeblockSUDO) received more data than size, truncating: size: %s, data len: %s, data: %s" % (size, str(len(data)), data))
        data = data[:size]

    (iounit, dummy, iooffset) = _alignData(size, offset)

    # Note that subprocess's "communicate" writes to the DD stdin in 512b chunks.
    # To avoid races with the pipe, ibs is set to 512 and "count" MUST NOT be used.
    cmd = [constants.EXT_DD, "oflag=%s" % DIRECTFLAG, "ibs=512",
            "obs=%d" % iounit, "seek=%d" % iooffset, "of=%s" % name]
    (rc, out, err) = execCmd(cmd, data=data, sudo=sudo)
    if rc:
        raise se.MiscBlockWriteException(name, offset, size)
    if not validateDDBytes(err, size):
        raise se.MiscBlockWriteIncomplete(name, offset, size)

    return out


def _alignData(length, offset):
    iounit = MEGA
    count = length
    iooffset = offset

    # Keep small IOps in single shot if possible
    if (length < MEGA) and (offset % length == 0) and (length % 512 == 0):
        # IO can be direct + single shot
        count = 1
        iounit = length
        iooffset = offset / iounit
        return (iounit, count, iooffset)

    # Compute largest chunk possible up to 1M for IO
    while iounit > 1:
        if (length >= iounit) and (offset % iounit == 0):
            count = length / iounit
            iooffset = offset / iounit
            break
        iounit = iounit >> 1

    return (iounit, count, iooffset)


def ddWatchCopy(src, dst, stop, idle, size, offset=0, sudo=False):
    """
    Copy src to dst using dd command with stop abilities
    """
    left = size
    baseoffset = offset

    try:
        int(size)
    except:
        raise se.InvalidParameterException("size", size)

    while left > 0:
        (iounit, count, iooffset) = _alignData(left, offset)
        oflag = None
        conv = "notrunc"
        if (iounit % 512) == 0:
            oflag = DIRECTFLAG
        else:
            conv += ",%s" % DATASYNCFLAG

        cmd = [constants.EXT_DD, "if=%s" % src, "of=%s" % dst, "bs=%d" % iounit,
               "seek=%s" % iooffset, "skip=%s" % iooffset, "conv=%s" % conv, 'count=%s' % count]

        if oflag:
            cmd.append("oflag=%s" % oflag)

        cmd = subprocess.list2cmdline(cmd)

        runAs = IOUSER
        if sudo:
            runAs = "root"

        cmd = [constants.EXT_IONICE, '-c2', '-n7', constants.EXT_SU, runAs, '-s', constants.EXT_SH, "-c", cmd]

        if not stop:
            (rc, out, err) = execCmd(cmd, sudo=True)
        else:
            (rc, out, err) = watchCmd(cmd, stop=stop, idle=idle, sudo=True)

        if rc:
            raise se.MiscBlockWriteException(dst, offset, size)

        if not validateDDBytes(err, iounit*count):
            raise se.MiscBlockWriteIncomplete(dst, offset, size)

        left = left % iounit
        offset = baseoffset + size - left

    return (rc, out, err)


def ddCopy(src, dst, size=None):
    """
    Copy src to dst using dd command
    """
    return ddWatchCopy(src, dst, None, None, size=size)


def parseBool(var):
    if isinstance(var,bool):
        return var
    # Transform: str -> bool
    if var.lower() == 'true':
        return True
    else:
        return False


def checksum(string, numBytes):
    bits = 8 * numBytes
    tmpArray = array('B')
    tmpArray.fromstring(string)
    csum = sum(tmpArray)
    return csum - (csum >> bits << bits)


def packUuid(s):
    s = ''.join([c for c in s if c != '-'])
    uuid = int(s, 16)
    high = uuid / 2**64
    low = uuid % 2**64
    # pack as 128bit little-endian <QQ
    return struct.pack('<QQ', low, high)


def unpackUuid(uuid):
    low, high = struct.unpack('<QQ', uuid)
    uuid = hex(low + 2**64 * high)[2:-1].rjust(STR_UUID_SIZE - 4,"0").lower() # remove leading 0x and trailing L
    s=""
    prev=0
    i=0
    for hypInd in UUID_HYPHENS:
        s+=uuid[prev:hypInd-i] + '-'
        prev=hypInd-i
        i+=1
    s+=uuid[prev:]
    return s #'-'.join([ s[0:8], s[8:12], s[12:16], s[16:20], s[20:] ])


def validateUUID(uuid, name="uuid"):
    """
    Ensure that uuid structure is 32 bytes long and is of the form: 8-4-4-4-12 (where each number depicts the amount of hex digits)
    """
    #FIXME : Why not use a regex? -> ^[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}$
    if len(uuid) != STR_UUID_SIZE:
        raise se.InvalidParameterException(name, uuid)
    for hypInd in UUID_HYPHENS:
        if uuid[hypInd] != '-':
            raise se.InvalidParameterException(name, uuid)
    try:
        #s=uuid[:8] + uuid[9:13] + uuid[14:18] + uuid[19:23] + uuid[24:]
        s=""
        prev=0
        for hypInd in UUID_HYPHENS:
            s+=uuid[prev:hypInd]
            prev=hypInd+1
        s+=uuid[prev:]
        tmp = int(s, 16)
    except:
        raise se.InvalidParameterException(name, uuid)
    return True

def validateInt(number, name): #FIXME: Consider using confutils validator?
    try:
        return int(number)
    except:
        raise se.InvalidParameterException(name, number)

def validateN(number, name):
    n = validateInt(number, name)
    if n < 0:
        raise se.InvalidParameterException(name, number)
    return n

def rotateFiles(dir, prefixName, gen, cp=False, persist=False):
    log.debug("(misc.rotateFiles) dir: %s, prefixName: %s, versions: %s" % (dir, prefixName, gen))
    gen = int(gen)
    files = ros.listdir(dir)
    files = [file for file in files if file.startswith(prefixName)] #FIXME: Why not use glob.glob?
    fd = {}
    for file in files:
        name = file.rsplit('.',1)
        try:
            ind = int(name[1])
        except ValueError:
            name[0] = file
            ind = 0
        except IndexError:
            ind = 0
        except:
            continue
        if ind < gen:
            fd[ind] = {'old': file, 'new': name[0] + '.' + str(ind+1)}

    keys = fd.keys()
    keys.sort(reverse=True)
    log.debug("(misc.rotateFiles) versions found: %s" % (keys))

    for key in keys:
        oldName = os.path.join(dir, fd[key]['old'])
        newName = os.path.join(dir, fd[key]['new'])
        if OVIRT_NODE and persist and not cp:
            try:
                execCmd([constants.EXT_UNPERSIST, oldName], logErr=False)
                execCmd([constants.EXT_UNPERSIST, newName], logErr=False)
            except:
                pass
        try:
            if cp:
                execCmd([constants.EXT_CP, oldName, newName])
                if OVIRT_NODE and persist and not ros.exists(newName):
                    execCmd([constants.EXT_PERSIST, newName], logErr=False)

            else:
                ros.rename(oldName, newName)
        except:
            pass
        if OVIRT_NODE and persist and not cp:
            try:
                execCmd([constants.EXT_PERSIST, newName], logErr=False)
            except:
                pass


def persistFile(name):
    if OVIRT_NODE:
        execCmd([constants.EXT_PERSIST, name])

def parseHumanReadableSize(size):
    #FIXME : Maybe use a regex -> ^(?P<num>\d+)(?P<sizeChar>[KkMmGgTt])$
    #FIXME : Why not support B and be done with it?
    if size.isdigit():
        # No suffix - pass it as is
        return int(size)

    size = size.upper()

    if size.endswith("T"):
        if size[:-1].isdigit():
            return int(size[:-1]) << 40

    if size.endswith("G"):
        if size[:-1].isdigit():
            return int(size[:-1]) << 30

    if size.endswith("M"):
        if size[:-1].isdigit():
            return int(size[:-1]) << 20

    if size.endswith("K"):
        if size[:-1].isdigit():
            return int(size[:-1]) << 10

    # Failing all the above we'd better just return 0
    return 0


def dmRemoveMapping(vgname):
    """
    Removes the mapping of the specified volume group.
    Utilizises the fact that the mapping created by the LVM looks like that
    e45c12b0--f520--498a--82bb--c6cb294b990f-master
    i.e vg name concatenated with volume name (dash is escaped with dash)
    """
    cmd = [constants.EXT_DMSETUP, "ls"]

    rc, out = execCmd(cmd)[0:2]

    if rc != 0:
        # report error
        return

    # convert the name to the style used in device mapper
    vgname = "--".join(vgname.split("-"))#FIXME : what's wrong with str.replace?

    for mapping in [i.split()[0] for i in out if i.startswith(vgname)]:
        cmd = [constants.EXT_DMSETUP, "remove", mapping]
        rc = execCmd(cmd)[0]
        if rc != 0:
            # report error
            pass

class _DisabledGcBlock(object):
    _refCount = 0
    _refLock = threading.Lock()
    _lastCollect = 0
    forceCollectInterval = 60

    def enter(self):
        self._refLock.acquire()
        try:
            if gc.isenabled():
                self._lastCollect = time.time()
                gc.disable()

            if (time.time() - self._lastCollect) > self.forceCollectInterval:
                self._lastCollect = time.time()
                gc.collect()

            self._refCount += 1
        finally:
            self._refLock.release()

    def exit(self):
        self._refLock.acquire()
        try:
            self._refCount -= 1
            if self._refCount == 0:
                gc.enable()
        finally:
            self._refLock.release()

disabledGcBlock = _DisabledGcBlock()


class DynamicBarrier(object):
    def __init__(self):
        self._lock = threading.Lock()
        self._cond = threading.Condition()

    def enter(self):
        """
        Enter the dynamic barrier. Returns True if you should be
        the one performing the operation. False if someone already
        did that for you.

        You only have to exit() if you actually entered.

        Example:

        >> if dynamicBarrier.enter():
        >>    print "Do stuff"
        >>    dynamicBarrier.exit()
        """
        self._cond.acquire()
        try:
            if self._lock.acquire(False):
                return True

            self._cond.wait()

            if self._lock.acquire(False):
                return True

            self._cond.wait()
            return False

        finally:
            self._cond.release()

    def exit(self):
        self._cond.acquire()
        try:
            self._lock.release()
            self._cond.notifyAll()
        finally:
            self._cond.release()

class samplingmethod(object):
    """
    This decorator purposefully doesn't support functions with
    parameters.
    In addition, if and exception is thrown, only the function running it will get the exception, the rest will get previous run results.
    Supporting parameters or exception passing to all functions would make the code much more complex for no reason.

    *** Don't use this decorator on recursive functions!!! ***
    """
    _log = logging.getLogger("SamplingMethod")
    def __init__(self, func):
        self.__func = func
        self.__funcName = func.func_code.co_name
        self.__lastResult = None
        self.__barrier = DynamicBarrier()

    def __call__(self, *args, **kwargs):
        self._log.debug("Trying to enter %s", self.__funcName)

        if self.__barrier.enter():
            self._log.debug("Performing %s", self.__funcName)

            try:
                self.__lastResult = self.__func(*args, **kwargs)
            finally:
                self.__barrier.exit()
        else:
            self._log.debug("No need to enter %s", self.__funcName)

        self._log.debug("Returning last result for %s", self.__funcName)

        return self.__lastResult

    def __get__(self, instance, _class=None):
        if instance is None:
            return self

        def newfunc(*args, **kwargs):
            return self.__func(instance, *args, **kwargs)

        return newfunc

class _DisabledGcBlock(object):
    _refCount = 0
    _refLock = threading.Lock()
    _lastCollect = 0
    forceCollectInterval = 60

    def enter(self):
        self._refLock.acquire()
        try:
            if gc.isenabled():
                self._lastCollect = time.time()
                gc.disable()

            if (time.time() - self._lastCollect) > self.forceCollectInterval:
                self._lastCollect = time.time()
                gc.collect()

            self._refCount += 1
        finally:
            self._refLock.release()

    def exit(self):
        self._refLock.acquire()
        try:
            self._refCount -= 1
            if self._refCount == 0:
                gc.enable()
        finally:
            self._refLock.release()

disabledGcBlock = _DisabledGcBlock()

def tmap(func, iterable):
    resultsDict = {}
    def wrapper(f, arg, index):
        try:
            resultsDict[index] = (f(arg), None)
        except Exception, e:
            resultsDict[index] = (None, e)

    threads = []
    for i, arg in enumerate(iterable):
        t = threading.Thread(target=wrapper, args=(func, arg, i))
        threads.append(t)
        t.start()

    for t in threads:
        t.join()

    results = [None] * len(resultsDict)
    for i, result in resultsDict.iteritems():
        results[i] = result

    finalResults = []
    for res, err in results:
        if err is not None:
            raise err
        finalResults.append(res)

    return tuple(finalResults)

def getfds():
    return [int(fd) for fd in os.listdir("/proc/self/fd")]
def pgrep(name):
    res = []
    for statFile in glob("/proc/*/status"):
        try:
            f = open(statFile, "r")
        except OSError:
            continue
        except IOError:
            continue

        try:
            try:
                procName = f.readline().split()[1]
                if procName == name:
                    res.append(int(os.path.basename(os.path.dirname(statFile))))
            except ValueError:
                continue
            except IOError:
                continue
        finally:
            f.close()
    return res

def getPidCmdline(pid):
    f = open("/proc/%d/cmdline" % pid , "r")
    try:
        return f.readline()
    finally:
        f.close()

def pidStat(pid):
    f = open("/proc/%d/stat" % pid , "r")
    try:
        return tuple(f.readline().split())
    finally:
        f.close()

# Upon import determine if we are running on ovirt
try:
    OVIRT_NODE = os.path.exists('/etc/rhev-hypervisor-release')
except:
    pass

